# @Time:2021/4/10 12:37
# @Author:andrew
# @email:zengjunjine1026@163.com
# @File:sys_mongo.py
# @veision 1.0
import multiprocessing

import pymongo
import redis
from settings import logger
from multiprocessing import pool

pool = redis.ConnectionPool(host='192.168.2.180', port=6379, decode_responses=True)
r = redis.Redis(connection_pool=pool)


class Source:
    def __init__(self, host, port, db, collection):
        self.host = host
        self.port = port
        self.db = db
        self.collection = collection


def transder_mongo_from_source_to_distination(source, destination):
    local_mongo = pymongo.MongoClient(source.host, source.port)
    db = local_mongo[source.db]
    collection = db[source.collection]
    save_collection = pymongo.MongoClient(destination.host, destination.port)[destination.db][destination.collection]
    redis_key = source.db + "_" + source.collection
    logger.info(redis_key)
    cusor = int(r.get(redis_key)) if r.exists(redis_key) else 0
    # cusor = 0
    res = collection.find().skip(cusor)
    for index, data in enumerate(res):
        logger.info(data)
        # del data['_id']
        r.set(redis_key, index + cusor)
        id = data['_id']
        del data['_id']
        # logger.info(index + cusor)
        save_collection.update_one({"_id": id}, {"$set": data}, upsert=True)


if __name__ == "__main__":
    host = "192.168.2.180"
    dest_host = "192.168.2.117"
    port = 27017
    client = pymongo.MongoClient(host,port)
    dbs = client.list_database_names()
    p_list = []
    for db in ['beike_house']:
        print(db)
        Mongo_db = client[db]
        collections = Mongo_db.list_collection_names()
        for collection in collections:
            source = Source(host,port,db,collection)
            destination = Source(dest_host,port,db,collection)
            p = multiprocessing.Process(target=transder_mongo_from_source_to_distination,args=(source,destination))
            p.start()
            p_list.append(p)

    # source = Source("manjaro", 27017, "jingdong_spiders", "comment_1")
    # destination = Source("qunhui", 27017, "jingdong_spiders", "comment_1")
    #
    # source2 = Source("manjaro", 27017, "jingdong_spiders", "comment_2")
    # source3 = Source("manjaro", 27017, "jingdong_spiders", "comment_3")
    # source5 = Source("manjaro", 27017, "jingdong_spiders", "comment_5")
    # destination2 = Source("qunhui", 27017, "jingdong_spiders", "comment_2")
    # destination3 = Source("qunhui", 27017, "jingdong_spiders", "comment_3")
    # destination5 = Source("qunhui", 27017, "jingdong_spiders", "comment_5")
    #
    # # source6 = Source("manjaro", 27017, "jingdong_spiders", "comment_3")
    # # transder_mongo_from_source_to_distination(source, destination)
    # p = multiprocessing.Process(target=transder_mongo_from_source_to_distination, args=(source, destination))
    # p2 = multiprocessing.Process(target=transder_mongo_from_source_to_distination, args=(source2, destination2))
    # p3 = multiprocessing.Process(target=transder_mongo_from_source_to_distination, args=(source3, destination3))
    # p5 = multiprocessing.Process(target=transder_mongo_from_source_to_distination, args=(source5, destination5))
    # p_lst = []
    # p.start()
    # # p.join()
    # p_lst.append(p)
    # p2.start()
    # # p2.join()
    # p_lst.append(p2)
    # p3.start()
    # # p3.join()
    # p_lst.append(p3)
    # p5.start()
    # # p5.join()
    # p_lst.append(p5)
    [p.join() for p in p_list]
    print("jfjj")